{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.04291243,  0.0479058 , -0.01014704,  0.02571197], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        #一局游戏最多走N步\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATkUlEQVR4nO3df2xT5b8H8He7rR2wnc4x1zq2Bq4aceGHOmA7+IdGKhMXI7I/1KBOQzBixxVniO5GUVEzg3/4E8c/CvyDeGeCxoUfzqEjamFS3c0YsGiCtwvQlh/fnW7TtV373D++d0crG9Kt9GnX9ys5CX2ep+3nPFvfnHOedjUIIQSIiCQwyi6AiDIXA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKSRFkBbt27F7NmzkZubi8rKSnR2dsoqhYgkkRJAn376KRoaGvDyyy/jp59+wsKFC1FdXQ2/3y+jHCKSxCDjw6iVlZVYvHgxPvjgAwBANBpFWVkZ1q9fjxdeeCHZ5RCRJNnJfsJQKAS3243Gxka9zWg0wuFwwOVyjXmfYDCIYDCo345Go7h48SJmzpwJg8Fw1WsmovgIITAwMICSkhIYjeOfaCU9gM6fP49IJAKr1RrTbrVacfLkyTHv09TUhFdffTUZ5RFRAvX19aG0tHTc/qQH0EQ0NjaioaFBv61pGux2O/r6+qAoisTKiGgsgUAAZWVlyM/Pv+y4pAdQUVERsrKy4PP5Ytp9Ph9sNtuY9zGbzTCbzZe0K4rCACJKYf90iSTpq2AmkwkVFRVob2/X26LRKNrb26GqarLLISKJpJyCNTQ0oK6uDosWLcKSJUvwzjvvYGhoCE888YSMcohIEikB9OCDD+LcuXPYtGkTvF4vbrnlFuzfv/+SC9NENLVJeR/QZAUCAVgsFmiaxmtARCnoSl+j/CwYEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpIk7gA4dOoT77rsPJSUlMBgM+Pzzz2P6hRDYtGkTrrvuOkybNg0OhwO//PJLzJiLFy9i9erVUBQFBQUFWLNmDQYHBye1I0SUfuIOoKGhISxcuBBbt24ds3/Lli147733sG3bNhw5cgQzZsxAdXU1hoeH9TGrV69GT08P2tra0NraikOHDuHJJ5+c+F4QUXoSkwBA7NmzR78djUaFzWYTb731lt7W398vzGaz+OSTT4QQQhw/flwAED/++KM+Zt++fcJgMIjTp09f0fNqmiYACE3TJlM+EV0lV/oaTeg1oFOnTsHr9cLhcOhtFosFlZWVcLlcAACXy4WCggIsWrRIH+NwOGA0GnHkyJExHzcYDCIQCMRsRJT+EhpAXq8XAGC1WmParVar3uf1elFcXBzTn52djcLCQn3M3zU1NcFisehbWVlZIssmIknSYhWssbERmqbpW19fn+ySiCgBEhpANpsNAODz+WLafT6f3mez2eD3+2P6R0ZGcPHiRX3M35nNZiiKErMRUfpLaADNmTMHNpsN7e3telsgEMCRI0egqioAQFVV9Pf3w+1262MOHjyIaDSKysrKRJZDRCkuO947DA4O4tdff9Vvnzp1Cl1dXSgsLITdbseGDRvw+uuv48Ybb8ScOXPw0ksvoaSkBCtXrgQA3Hzzzbjnnnuwdu1abNu2DeFwGPX19XjooYdQUlKSsB0jojQQ7/LaN998IwBcstXV1Qkh/r0U/9JLLwmr1SrMZrNYtmyZ6O3tjXmMCxcuiIcffljk5eUJRVHEE088IQYGBhK+xEdEclzpa9QghBAS829CAoEALBYLNE3j9SCiFHSlr9G0WAUjoqmJAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJE/fX8hAlmohG0O85hmh4WG+bYf0PmPOLYDAYJFZGVxsDiKSLRkbQ5/pvBAPn9LbZd9TBfFORxKooGXgKRqlJRGVXQEnAAKKUlIZfV0cTwACiFMUAygQMIEpNPALKCAwgSkk8BcsMDCBKTQygjBBXADU1NWHx4sXIz89HcXExVq5cid7e3pgxw8PDcDqdmDlzJvLy8lBbWwufzxczxuPxoKamBtOnT0dxcTE2btyIkZGRye8NTRmCq2AZIa4A6ujogNPpxOHDh9HW1oZwOIzly5djaGhIH/Pss8/iyy+/REtLCzo6OnDmzBmsWrVK749EIqipqUEoFMIPP/yAnTt3YseOHdi0aVPi9oqmAB4BZQQxCX6/XwAQHR0dQggh+vv7RU5OjmhpadHHnDhxQgAQLpdLCCHE3r17hdFoFF6vVx/T3NwsFEURwWDwip5X0zQBQGiaNpnyKUWMhIbF/+z6L9G5ba2+nek6IKLRqOzSaIKu9DU6qWtAmqYBAAoLCwEAbrcb4XAYDodDHzN37lzY7Xa4XC4AgMvlwvz582G1WvUx1dXVCAQC6OnpGfN5gsEgAoFAzEZTHK8BZYQJB1A0GsWGDRtw++23Y968eQAAr9cLk8mEgoKCmLFWqxVer1cf89fwGe0f7RtLU1MTLBaLvpWVlU20bEoXvAaUESYcQE6nE8eOHcPu3bsTWc+YGhsboWmavvX19V315yS5BK8BZYQJfRi1vr4era2tOHToEEpLS/V2m82GUCiE/v7+mKMgn88Hm82mj+ns7Ix5vNFVstExf2c2m2E2mydSKqUrnoJlhLiOgIQQqK+vx549e3Dw4EHMmTMnpr+iogI5OTlob2/X23p7e+HxeKCqKgBAVVV0d3fD7/frY9ra2qAoCsrLyyezLzSFCAZQRojrCMjpdGLXrl344osvkJ+fr1+zsVgsmDZtGiwWC9asWYOGhgYUFhZCURSsX78eqqqiqqoKALB8+XKUl5fj0UcfxZYtW+D1evHiiy/C6XTyKIf+xADKCHEFUHNzMwDgzjvvjGnfvn07Hn/8cQDA22+/DaPRiNraWgSDQVRXV+PDDz/Ux2ZlZaG1tRXr1q2DqqqYMWMG6urqsHnz5sntCU0pfCNiZjCINDzWDQQCsFgs0DQNiqLILocmKRIOouezzTF/kOy6W+/FrMX38y8ipqkrfY3ys2CUotLu/0WaAAYQpaQ0PDCnCWAAUWpiAGUEBhClJgZQRmAAUUriKlhmYACRfAYDYIj9VRQiIqkYSiYGEElnMBiRa4n9gPIf/zorqRpKJgYQSWcwGGDMin1PrIjwCCgTMIAoNfANhxmJAUSpgQGUkRhAlBIMYABlIgYQpQYeAWUkBhClBH7oNDMxgCg1MIAyEgOIUoAB/FXMTPypU0rgKVhmYgBRamAAZSQGEKUGBlBGYgCRfAa+DyhTMYAoNfAIKCMxgCgl8CJ0ZmIAUYpgAGUiBhClgEv/IBllBv7UKSXwFCwzxfXNqEQTFQ6HMTQ0NHanEAiGQjFNkcgItP7+cS9Om81mTJs2LcFVUrIxgCgpvv/+ezzyyCPj9j+27CasVK/Xbx896saK/5w/7tcTOp1ONDY2JrhKSjYGECVFMBjE6dOnx+3XAqXoC87F2eD1sGSfw1BwH06fPj1uAAUCgatTKCVVXNeAmpubsWDBAiiKAkVRoKoq9u3bp/cPDw/D6XRi5syZyMvLQ21tLXw+X8xjeDwe1NTUYPr06SguLsbGjRsxMjKSmL2htHV6+HqcGFyKi+FZOPXHQvQOVfLLmTNAXAFUWlqKN998E263G0ePHsVdd92F+++/Hz09PQCAZ599Fl9++SVaWlrQ0dGBM2fOYNWqVfr9I5EIampqEAqF8MMPP2Dnzp3YsWMHNm3alNi9orQzOKIgqh+QGzAYKQCX5qe+uE7B7rvvvpjbb7zxBpqbm3H48GGUlpbio48+wq5du3DXXXcBALZv346bb74Zhw8fRlVVFb766iscP34cX3/9NaxWK2655Ra89tpreP755/HKK6/AZDIlbs8orRTl/C9yjQMYjuYhyxDGLPOvAI+BprwJXwOKRCJoaWnB0NAQVFWF2+1GOByGw+HQx8ydOxd2ux0ulwtVVVVwuVyYP38+rNY/vwOquroa69atQ09PD2699da4ajh58iTy8vImuguURB6P57L9vb/8jHB4My6EZiEv+18YGfzlsuPPnz+P48ePJ7JESqDBwcErGhd3AHV3d0NVVQwPDyMvLw979uxBeXk5urq6YDKZUFBQEDPearXC6/UCALxeb0z4jPaP9o0nGAwiGAzqt0cvQGqaxutHaWLcJfj/1/WrF12/jv878HfBYBD9/f2TrIquln/6eY+KO4BuuukmdHV1QdM0fPbZZ6irq0NHR0fcBcajqakJr7766iXtlZWVUBTlqj43JcbAwEBCH2/WrFlYunRpQh+TEudKVynjfie0yWTCDTfcgIqKCjQ1NWHhwoV49913YbPZEAqFLvlfyefzwWazAQBsNtslq2Kjt0fHjKWxsRGapulbX19fvGUTUQqa9EcxotEogsEgKioqkJOTg/b2dr2vt7cXHo8HqqoCAFRVRXd3N/x+vz6mra0NiqKgvLx83Ocwm8360v/oRkTpL65TsMbGRqxYsQJ2ux0DAwPYtWsXvv32Wxw4cAAWiwVr1qxBQ0MDCgsLoSgK1q9fD1VVUVVVBQBYvnw5ysvL8eijj2LLli3wer148cUX4XQ6YTabr8oOElHqiiuA/H4/HnvsMZw9exYWiwULFizAgQMHcPfddwMA3n77bRiNRtTW1iIYDKK6uhoffvihfv+srCy0trZi3bp1UFUVM2bMQF1dHTZv3pzYvaKUk52dndAjV/6HNTUYhBBp92aLQCAAi8UCTdN4OpYmhoeHce7cuYQ9Xn5+/iUrrpQ6rvQ1ys+CUVLk5uairKxMdhmUYvj3gIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE02bILmAghBAAgEAhIroSIxjL62hx9rY4nLQPowoULAICysjLJlRDR5QwMDMBisYzbn5YBVFhYCADweDyX3TmKFQgEUFZWhr6+PiiKIructMA5mxghBAYGBlBSUnLZcWkZQEbjvy9dWSwW/lJMgKIonLc4cc7idyUHB7wITUTSMICISJq0DCCz2YyXX34ZZrNZdilphfMWP87Z1WUQ/7RORkR0laTlERARTQ0MICKShgFERNIwgIhImrQMoK1bt2L27NnIzc1FZWUlOjs7ZZckTVNTExYvXoz8/HwUFxdj5cqV6O3tjRkzPDwMp9OJmTNnIi8vD7W1tfD5fDFjPB4PampqMH36dBQXF2Pjxo0YGRlJ5q5I8+abb8JgMGDDhg16G+csSUSa2b17tzCZTOLjjz8WPT09Yu3ataKgoED4fD7ZpUlRXV0ttm/fLo4dOya6urrEvffeK+x2uxgcHNTHPPXUU6KsrEy0t7eLo0ePiqqqKrF06VK9f2RkRMybN084HA7x888/i71794qioiLR2NgoY5eSqrOzU8yePVssWLBAPPPMM3o75yw50i6AlixZIpxOp347EomIkpIS0dTUJLGq1OH3+wUA0dHRIYQQor+/X+Tk5IiWlhZ9zIkTJwQA4XK5hBBC7N27VxiNRuH1evUxzc3NQlEUEQwGk7sDSTQwMCBuvPFG0dbWJu644w49gDhnyZNWp2ChUAhutxsOh0NvMxqNcDgccLlcEitLHZqmAfjzA7tutxvhcDhmzubOnQu73a7Pmcvlwvz582G1WvUx1dXVCAQC6OnpSWL1yeV0OlFTUxMzNwDnLJnS6sOo58+fRyQSifmhA4DVasXJkyclVZU6otEoNmzYgNtvvx3z5s0DAHi9XphMJhQUFMSMtVqt8Hq9+pix5nS0byravXs3fvrpJ/z444+X9HHOkietAoguz+l04tixY/juu+9kl5LS+vr68Mwzz6CtrQ25ubmyy8loaXUKVlRUhKysrEtWI3w+H2w2m6SqUkN9fT1aW1vxzTffoLS0VG+32WwIhULo7++PGf/XObPZbGPO6WjfVON2u+H3+3HbbbchOzsb2dnZ6OjowHvvvYfs7GxYrVbOWZKkVQCZTCZUVFSgvb1db4tGo2hvb4eqqhIrk0cIgfr6euzZswcHDx7EnDlzYvorKiqQk5MTM2e9vb3weDz6nKmqiu7ubvj9fn1MW1sbFEVBeXl5cnYkiZYtW4bu7m50dXXp26JFi7B69Wr935yzJJF9FTxeu3fvFmazWezYsUMcP35cPPnkk6KgoCBmNSKTrFu3TlgsFvHtt9+Ks2fP6tvvv/+uj3nqqaeE3W4XBw8eFEePHhWqqgpVVfX+0SXl5cuXi66uLrF//35x7bXXZtSS8l9XwYTgnCVL2gWQEEK8//77wm63C5PJJJYsWSIOHz4suyRpAIy5bd++XR/zxx9/iKefflpcc801Yvr06eKBBx4QZ8+ejXmc3377TaxYsUJMmzZNFBUVieeee06Ew+Ek7408fw8gzlly8M9xEJE0aXUNiIimFgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkzf8BHktyCHp+ynYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)\n",
      "env.action_space= Discrete(2)\n",
      "state= [ 0.04587445 -0.02671207  0.03698547  0.020224  ]\n",
      "action= 1\n",
      "next_state= [ 0.04534021  0.1678605   0.03738995 -0.260564  ]\n",
      "reward= 1.0\n",
      "done= False\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=2, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=2, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#计算动作的模型,也是真正要用的模型\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    ")\n",
    "\n",
    "#经验网络,用于评估一个状态的分数\n",
    "next_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    ")\n",
    "\n",
    "#把model的参数复制给next_model\n",
    "next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "model, next_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "#得到一个动作\n",
    "def get_action(state):\n",
    "    if random.random() < 0.01:\n",
    "        return random.choice([0, 1])\n",
    "\n",
    "    #走神经网络,得到一个动作\n",
    "    state = torch.FloatTensor(state).reshape(1, 4)\n",
    "\n",
    "    return model(state).argmax().item()\n",
    "\n",
    "\n",
    "get_action([0.0013847, -0.01194451, 0.04260966, 0.00688801])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 1_0000:\n",
    "        datas.pop(0)\n",
    "\n",
    "\n",
    "update_data()\n",
    "\n",
    "len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_41739/2021397730.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:230.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[ 6.9080e-03,  8.3188e-01, -1.0945e-01, -1.2396e+00],\n",
       "         [ 1.1659e-01,  5.7994e-01, -1.6258e-01, -1.0571e+00],\n",
       "         [-2.2631e-02,  5.8258e-01,  2.7247e-02, -8.3719e-01],\n",
       "         [ 9.0803e-02,  7.6052e-01, -1.7939e-01, -1.3275e+00],\n",
       "         [-2.2402e-02,  8.2984e-01, -6.7245e-02, -1.1907e+00],\n",
       "         [-4.4527e-02,  1.8322e-01,  3.0115e-02, -2.8778e-01],\n",
       "         [ 9.5200e-02,  8.0116e-01, -1.3847e-01, -1.2238e+00],\n",
       "         [ 4.6056e-03,  2.0080e-01,  4.5945e-02, -3.0775e-01],\n",
       "         [ 1.2549e-01,  7.8756e-01, -1.9666e-01, -1.3698e+00],\n",
       "         [ 1.3037e-01,  7.9307e-01, -1.6008e-01, -1.2802e+00],\n",
       "         [ 1.0429e-01,  7.3912e-01, -1.6689e-01, -1.3640e+00],\n",
       "         [ 4.0308e-02,  7.8627e-01, -4.9682e-02, -1.1381e+00],\n",
       "         [ 1.3519e-01,  6.3556e-01, -9.7604e-02, -8.5991e-01],\n",
       "         [ 1.0113e-01,  7.7309e-01, -1.3650e-01, -1.3041e+00],\n",
       "         [ 1.1122e-01,  6.0807e-01, -1.6295e-01, -9.7753e-01],\n",
       "         [-4.8710e-02,  2.4288e-01, -3.2508e-02, -2.7564e-01],\n",
       "         [ 4.5671e-03,  5.8206e-01, -1.1920e-02, -8.2522e-01],\n",
       "         [ 7.5918e-04,  5.5682e-01, -3.6920e-02, -8.2766e-01],\n",
       "         [ 9.3439e-02,  5.4239e-01, -1.4631e-01, -1.0291e+00],\n",
       "         [-3.3304e-02,  5.7267e-01,  1.2943e-02, -8.5572e-01],\n",
       "         [ 3.8696e-02,  4.3850e-01,  2.7650e-02, -5.2137e-01],\n",
       "         [ 4.4781e-03,  6.3728e-03,  4.6548e-02, -3.0108e-02],\n",
       "         [-4.9656e-02,  4.7298e-02, -3.3053e-02,  2.7285e-02],\n",
       "         [ 6.0375e-02,  5.5868e-01, -3.9472e-02, -8.5297e-01],\n",
       "         [-1.4276e-02,  7.5175e-01, -1.4606e-02, -1.1157e+00],\n",
       "         [-4.1103e-02,  2.1240e-01,  4.9929e-02, -2.6140e-01],\n",
       "         [ 2.7148e-02,  6.0087e-01, -4.2216e-02, -8.0766e-01],\n",
       "         [ 2.0313e-02,  5.7351e-01, -6.7197e-02, -8.7547e-01],\n",
       "         [ 3.3737e-02, -4.9446e-02, -4.0403e-02,  1.1750e-03],\n",
       "         [ 7.6238e-02,  7.6123e-01, -1.1970e-01, -1.2435e+00],\n",
       "         [ 1.3846e-01,  7.9082e-01, -1.5509e-01, -1.3015e+00],\n",
       "         [ 1.0601e-01,  5.6805e-01, -2.0594e-01, -1.0959e+00],\n",
       "         [ 1.7841e-01,  6.4203e-01, -1.5882e-01, -1.0103e+00],\n",
       "         [-7.7729e-04,  6.0053e-01, -4.3411e-03, -7.9985e-01],\n",
       "         [ 3.5860e-02,  5.7331e-01, -3.1673e-02, -8.9887e-01],\n",
       "         [ 9.0775e-02,  5.9347e-01, -9.9566e-02, -8.7828e-01],\n",
       "         [-1.0979e-02,  7.7732e-01,  1.0503e-02, -1.1212e+00],\n",
       "         [ 9.6669e-04,  5.9066e-01,  6.1951e-03, -8.3392e-01],\n",
       "         [ 2.8494e-02,  5.9072e-01, -3.2976e-02, -8.3528e-01],\n",
       "         [-1.1093e-02,  6.1882e-03,  2.1860e-02,  2.4570e-02],\n",
       "         [ 7.1488e-02,  5.8980e-01, -5.2272e-02, -8.6631e-01],\n",
       "         [ 3.2835e-02,  4.9205e-02,  3.1562e-02,  4.3466e-02],\n",
       "         [ 4.4848e-02,  3.9371e-01, -2.4118e-02, -6.3823e-01],\n",
       "         [ 3.2748e-02,  1.4623e-01, -4.0380e-02, -3.0398e-01],\n",
       "         [-3.5980e-02,  1.6684e-01,  1.7522e-02, -2.4783e-01],\n",
       "         [ 9.5508e-02,  7.9047e-01, -1.3145e-01, -1.2387e+00],\n",
       "         [ 2.3546e-02,  6.3832e-01, -1.3425e-01, -9.8316e-01],\n",
       "         [ 6.3192e-02,  5.9162e-01, -5.9945e-02, -8.3506e-01],\n",
       "         [ 1.9125e-01,  4.4934e-01, -1.7903e-01, -7.7144e-01],\n",
       "         [-3.5419e-02, -2.8036e-02,  1.6731e-02,  3.9532e-02],\n",
       "         [ 1.6526e-02,  5.8978e-01,  2.8079e-02, -8.6548e-01],\n",
       "         [ 1.5428e-01,  5.9797e-01, -1.8112e-01, -1.0611e+00],\n",
       "         [ 4.7326e-02,  7.6885e-01, -4.9650e-02, -1.2013e+00],\n",
       "         [ 1.1811e-01,  5.7185e-01, -1.9089e-01, -1.0879e+00],\n",
       "         [ 5.8684e-02,  7.7196e-01, -1.2695e-01, -1.2482e+00],\n",
       "         [-2.8720e-02,  6.0124e-01,  3.3943e-02, -8.1619e-01],\n",
       "         [ 1.2819e-01,  7.7680e-01, -1.8372e-01, -1.3960e+00],\n",
       "         [ 4.2511e-02,  5.3764e-01, -5.8641e-02, -9.1606e-01],\n",
       "         [ 8.6635e-02,  5.5997e-01, -7.9687e-02, -8.8337e-01],\n",
       "         [-3.4262e-02,  1.9355e-01,  4.3951e-02, -2.7834e-01],\n",
       "         [ 5.3011e-02,  6.4208e-01, -1.8021e-01, -1.0740e+00],\n",
       "         [ 6.2703e-02,  5.7440e-01, -7.3677e-02, -9.2462e-01],\n",
       "         [ 3.8105e-02,  7.5412e-01, -9.3233e-02, -1.1719e+00],\n",
       "         [-3.4244e-02, -9.1716e-04,  4.3947e-02,  1.6049e-04]]),\n",
       " tensor([[0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1]]),\n",
       " tensor([[1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.]]),\n",
       " tensor([[ 2.3546e-02,  6.3832e-01, -1.3425e-01, -9.8316e-01],\n",
       "         [ 1.2819e-01,  7.7680e-01, -1.8372e-01, -1.3960e+00],\n",
       "         [-1.0979e-02,  7.7732e-01,  1.0503e-02, -1.1212e+00],\n",
       "         [ 1.0601e-01,  5.6805e-01, -2.0594e-01, -1.0959e+00],\n",
       "         [-5.8051e-03,  6.3565e-01, -9.1058e-02, -9.1979e-01],\n",
       "         [-4.0862e-02,  3.7790e-01,  2.4359e-02, -5.7081e-01],\n",
       "         [ 1.1122e-01,  6.0807e-01, -1.6295e-01, -9.7753e-01],\n",
       "         [ 8.6215e-03,  3.9524e-01,  3.9790e-02, -5.8560e-01],\n",
       "         [ 1.4124e-01,  5.9536e-01, -2.2405e-01, -1.1445e+00],\n",
       "         [ 1.4623e-01,  6.0031e-01, -1.8568e-01, -1.0416e+00],\n",
       "         [ 1.1907e-01,  5.4644e-01, -1.9417e-01, -1.1278e+00],\n",
       "         [ 5.6034e-02,  5.9184e-01, -7.2445e-02, -8.6145e-01],\n",
       "         [ 1.4790e-01,  4.4190e-01, -1.1480e-01, -5.9944e-01],\n",
       "         [ 1.1659e-01,  5.7994e-01, -1.6258e-01, -1.0571e+00],\n",
       "         [ 1.2338e-01,  8.0495e-01, -1.8250e-01, -1.3166e+00],\n",
       "         [-4.3852e-02,  4.3845e-01, -3.8020e-02, -5.7840e-01],\n",
       "         [ 1.6208e-02,  7.7734e-01, -2.8425e-02, -1.1216e+00],\n",
       "         [ 1.1896e-02,  7.5243e-01, -5.3473e-02, -1.1317e+00],\n",
       "         [ 1.0429e-01,  7.3912e-01, -1.6689e-01, -1.3640e+00],\n",
       "         [-2.1851e-02,  7.6762e-01, -4.1717e-03, -1.1443e+00],\n",
       "         [ 4.7466e-02,  6.3323e-01,  1.7222e-02, -8.0522e-01],\n",
       "         [ 4.6056e-03,  2.0080e-01,  4.5945e-02, -3.0775e-01],\n",
       "         [-4.8710e-02,  2.4288e-01, -3.2508e-02, -2.7564e-01],\n",
       "         [ 7.1549e-02,  7.5431e-01, -5.6531e-02, -1.1578e+00],\n",
       "         [ 7.5918e-04,  5.5682e-01, -3.6920e-02, -8.2766e-01],\n",
       "         [-3.6855e-02,  4.0677e-01,  4.4701e-02, -5.3792e-01],\n",
       "         [ 3.9165e-02,  7.9654e-01, -5.8369e-02, -1.1133e+00],\n",
       "         [ 3.1783e-02,  7.6948e-01, -8.4706e-02, -1.1885e+00],\n",
       "         [ 3.2748e-02,  1.4623e-01, -4.0380e-02, -3.0398e-01],\n",
       "         [ 9.1463e-02,  5.6783e-01, -1.4457e-01, -9.9058e-01],\n",
       "         [ 1.5428e-01,  5.9797e-01, -1.8112e-01, -1.0611e+00],\n",
       "         [ 1.1737e-01,  7.6520e-01, -2.2785e-01, -1.4455e+00],\n",
       "         [ 1.9125e-01,  4.4934e-01, -1.7903e-01, -7.7144e-01],\n",
       "         [ 1.1233e-02,  7.9571e-01, -2.0338e-02, -1.0939e+00],\n",
       "         [ 4.7326e-02,  7.6885e-01, -4.9650e-02, -1.2013e+00],\n",
       "         [ 1.0264e-01,  7.8979e-01, -1.1713e-01, -1.2005e+00],\n",
       "         [ 4.5671e-03,  5.8206e-01, -1.1920e-02, -8.2522e-01],\n",
       "         [ 1.2780e-02,  7.8570e-01, -1.0483e-02, -1.1247e+00],\n",
       "         [ 4.0308e-02,  7.8627e-01, -4.9682e-02, -1.1381e+00],\n",
       "         [-1.0969e-02,  2.0099e-01,  2.2351e-02, -2.6114e-01],\n",
       "         [ 8.3284e-02,  7.8560e-01, -6.9598e-02, -1.1750e+00],\n",
       "         [ 3.3819e-02,  2.4386e-01,  3.2432e-02, -2.3909e-01],\n",
       "         [ 5.2722e-02,  5.8916e-01, -3.6883e-02, -9.3841e-01],\n",
       "         [ 3.5673e-02,  3.4190e-01, -4.6459e-02, -6.0912e-01],\n",
       "         [-3.2643e-02,  3.6171e-01,  1.2565e-02, -5.3493e-01],\n",
       "         [ 1.1132e-01,  5.9726e-01, -1.5622e-01, -9.8995e-01],\n",
       "         [ 3.6312e-02,  8.3496e-01, -1.5391e-01, -1.3148e+00],\n",
       "         [ 7.5025e-02,  7.8751e-01, -7.6646e-02, -1.1460e+00],\n",
       "         [ 2.0024e-01,  6.4642e-01, -1.9446e-01, -1.1147e+00],\n",
       "         [-3.5980e-02,  1.6684e-01,  1.7522e-02, -2.4783e-01],\n",
       "         [ 2.8322e-02,  7.8451e-01,  1.0769e-02, -1.1492e+00],\n",
       "         [ 1.6624e-01,  4.0565e-01, -2.0234e-01, -8.3028e-01],\n",
       "         [ 6.2703e-02,  5.7440e-01, -7.3677e-02, -9.2462e-01],\n",
       "         [ 1.2955e-01,  7.6891e-01, -2.1264e-01, -1.4339e+00],\n",
       "         [ 7.4123e-02,  5.7867e-01, -1.5191e-01, -9.9786e-01],\n",
       "         [-1.6695e-02,  7.9588e-01,  1.7619e-02, -1.0980e+00],\n",
       "         [ 1.4373e-01,  5.8437e-01, -2.1164e-01, -1.1660e+00],\n",
       "         [ 5.3264e-02,  7.3351e-01, -7.6963e-02, -1.2266e+00],\n",
       "         [ 9.7834e-02,  7.5608e-01, -9.7354e-02, -1.2000e+00],\n",
       "         [-3.0391e-02,  3.8802e-01,  3.8384e-02, -5.5684e-01],\n",
       "         [ 6.5853e-02,  4.4974e-01, -2.0169e-01, -8.4284e-01],\n",
       "         [ 7.4191e-02,  7.7044e-01, -9.2169e-02, -1.2395e+00],\n",
       "         [ 5.3187e-02,  5.6032e-01, -1.1667e-01, -9.0983e-01],\n",
       "         [-3.4262e-02,  1.9355e-01,  4.3951e-02, -2.7834e-01]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 4]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 4]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state, action, reward, next_state, over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #使用状态计算出动作的logits\n",
    "    #[b, 4] -> [b, 2]\n",
    "    value = model(state)\n",
    "\n",
    "    #根据实际使用的action取出每一个值\n",
    "    #这个值就是模型评估的在该状态下,执行动作的分数\n",
    "    #在执行动作前,显然并不知道会得到的反馈和next_state\n",
    "    #所以这里不能也不需要考虑next_state和reward\n",
    "    #[b, 2] -> [b, 1]\n",
    "    value = value.gather(dim=1, index=action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #上面已经把模型认为的状态下执行动作的分数给评估出来了\n",
    "    #下面使用next_state和reward计算真实的分数\n",
    "    #针对一个状态,它到底应该多少分,可以使用以往模型积累的经验评估\n",
    "    #这也是没办法的办法,因为显然没有精确解,这里使用延迟更新的next_model评估\n",
    "\n",
    "    #使用next_state计算下一个状态的分数\n",
    "    #[b, 4] -> [b, 2]\n",
    "    with torch.no_grad():\n",
    "        target = next_model(next_state)\n",
    "\n",
    "    #取所有动作中分数最大的\n",
    "    #[b, 2] -> [b, 1]\n",
    "    target = target.max(dim=1)[0]\n",
    "    target = target.reshape(-1, 1)\n",
    "\n",
    "    #下一个状态的分数乘以一个系数,相当于权重\n",
    "    target *= 0.98\n",
    "\n",
    "    #如果next_state已经游戏结束,则next_state的分数是0\n",
    "    #因为如果下一步已经游戏结束,显然不需要再继续玩下去,也就不需要考虑next_state了.\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #加上reward就是最终的分数\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.8"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play=False):\n",
    "    state = env.reset()\n",
    "    reward_sum = 0\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = get_action(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "sum([test() for _ in range(20)]) / 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-3)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(500):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算一批样本的value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(reward, next_state, over)\n",
    "\n",
    "            #更新参数\n",
    "            loss = loss_fn(value, target)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            #把model的参数复制给next_model\n",
    "            if (i + 1) % 10 == 0:\n",
    "                next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 50 == 0:\n",
    "            print(epoch, len(datas), sum([test() for _ in range(5)]) / 5)\n",
    "\n",
    "    torch.save(model, 'save/4.DQN_CartPole')\n",
    "\n",
    "\n",
    "#train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAASsUlEQVR4nO3df0xTd78H8DcVWkU4RWS0I9DIMjPl+mtDhaO52aJ9ZI6YOfljM8Yxr9HIihFZzEbij7kfwegfbm6KuTcb+o+6sIQtEn+MgeJdrKI47oOo3LnHPRCl7dTQAkop9Hv/eMK56wRHAful+H4lJ/Gc76ft53zxvNOe0wMRQggBIiIJdLIbIKKnFwOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikkRZA+/fvx5QpUzB+/HhkZGSgtrZWVitEJImUAPrmm29QWFiIHTt24MqVK5g9ezaysrLgcrlktENEkkTIuBk1IyMD8+bNw5dffgkA8Pv9SElJwcaNG/HBBx+Euh0ikiQy1C/Y3d2Nuro6FBUVadt0Oh2sVivsdnu/j/F6vfB6vdq63+/H/fv3MXnyZERERDzxnokoOEIItLe3IykpCTrdwB+0Qh5Ad+/eRW9vL0wmU8B2k8mEGzdu9PuY4uJi7Ny5MxTtEdEIamlpQXJy8oDjIQ+goSgqKkJhYaG27na7YbFY0NLSAkVRJHZGRP3xeDxISUlBbGzsY+tCHkAJCQkYN24cnE5nwHan0wmz2dzvYwwGAwwGwyPbFUVhABGNYn91iiTkV8H0ej3S09NRVVWlbfP7/aiqqoKqqqFuh4gkkvIRrLCwELm5uZg7dy7mz5+Pzz77DJ2dnVizZo2MdohIEikB9Oabb+L333/H9u3b4XA4MGfOHJw6deqRE9NENLZJ+R7QcHk8HhiNRrjdbp4DIhqFBnuM8l4wIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARSRN0AJ07dw7Lli1DUlISIiIi8N133wWMCyGwfft2PPvss5gwYQKsVit++eWXgJr79+9j1apVUBQFcXFxWLt2LTo6Ooa1I0QUfoIOoM7OTsyePRv79+/vd3z37t3Yt28fDh48iIsXL2LixInIyspCV1eXVrNq1So0NjaisrISFRUVOHfuHNavXz/0vSCi8CSGAYAoLy/X1v1+vzCbzWLPnj3atra2NmEwGMTRo0eFEEJcu3ZNABCXLl3Sak6ePCkiIiLE7du3B/W6brdbABBut3s47RPREzLYY3REzwHdunULDocDVqtV22Y0GpGRkQG73Q4AsNvtiIuLw9y5c7Uaq9UKnU6Hixcv9vu8Xq8XHo8nYCGi8DeiAeRwOAAAJpMpYLvJZNLGHA4HEhMTA8YjIyMRHx+v1fxZcXExjEajtqSkpIxk20QkSVhcBSsqKoLb7daWlpYW2S0R0QgY0QAym80AAKfTGbDd6XRqY2azGS6XK2C8p6cH9+/f12r+zGAwQFGUgIWIwt+IBlBqairMZjOqqqq0bR6PBxcvXoSqqgAAVVXR1taGuro6raa6uhp+vx8ZGRkj2Q4RjXKRwT6go6MDN2/e1NZv3bqF+vp6xMfHw2KxoKCgAJ988gmmTp2K1NRUbNu2DUlJSVi+fDkAYPr06Xj11Vexbt06HDx4ED6fD/n5+XjrrbeQlJQ0YjtGRGEg2MtrZ86cEQAeWXJzc4UQ/7oUv23bNmEymYTBYBCLFy8WTU1NAc9x7949sXLlShETEyMURRFr1qwR7e3tI36Jj4jkGOwxGiGEEBLzb0g8Hg+MRiPcbjfPBxGNQoM9RsPiKhgRjU0MICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEiaoP8sD8nT7riJ7vZ72rpBeQYxpuckdkQ0PAygMOK6egb3f72krSe8sIABRGGNH8GISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNEEFUHFxMebNm4fY2FgkJiZi+fLlaGpqCqjp6uqCzWbD5MmTERMTg5ycHDidzoCa5uZmZGdnIzo6GomJidiyZQt6enqGvzdEFFaCCqCamhrYbDZcuHABlZWV8Pl8WLJkCTo7O7WazZs34/jx4ygrK0NNTQ3u3LmDFStWaOO9vb3Izs5Gd3c3zp8/j8OHD+PQoUPYvn37yO0VEYUHMQwul0sAEDU1NUIIIdra2kRUVJQoKyvTaq5fvy4ACLvdLoQQ4sSJE0Kn0wmHw6HVlJSUCEVRhNfrHdTrut1uAUC43e7htB92blb+p6g9uE5b/nGmVHZLRP0a7DE6rHNAbrcbABAfHw8AqKurg8/ng9Vq1WqmTZsGi8UCu90OALDb7Zg5cyZMJpNWk5WVBY/Hg8bGxn5fx+v1wuPxBCxEFP6GHEB+vx8FBQVYuHAhZsyYAQBwOBzQ6/WIi4sLqDWZTHA4HFrNH8Onb7xvrD/FxcUwGo3akpKSMtS2iWgUGXIA2Ww2XL16FceOHRvJfvpVVFQEt9utLS0tLU/8NYnoyRvSzaj5+fmoqKjAuXPnkJycrG03m83o7u5GW1tbwLsgp9MJs9ms1dTW1gY8X99Vsr6aPzMYDDAYDENplYhGsaDeAQkhkJ+fj/LyclRXVyM1NTVgPD09HVFRUaiqqtK2NTU1obm5GaqqAgBUVUVDQwNcLpdWU1lZCUVRkJaWNpx9IaIwE9Q7IJvNhiNHjuD7779HbGysds7GaDRiwoQJMBqNWLt2LQoLCxEfHw9FUbBx40aoqorMzEwAwJIlS5CWlobVq1dj9+7dcDgc2Lp1K2w2G9/lED1lggqgkpISAMArr7wSsL20tBTvvPMOAGDv3r3Q6XTIycmB1+tFVlYWDhw4oNWOGzcOFRUVyMvLg6qqmDhxInJzc/HRRx8Nb0+IKOxECCGE7CaC5fF4YDQa4Xa7oSiK7HZC5tcf/+uRX0iW+so78hoiGsBgj1HeC0ZE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpImqL+MSk+W1+vFw4cPBxz3+XwB693d3WhraxuwfsKECfxz1zSqMYBGkaNHj2Lr1q0Djm9+Yw7+/d+StPXy8nJ8+R8D/0nrPXv2YOXKlSPaI9FIYgCNIh0dHbh9+/bA4w/S8OvDObjnS8YzUS3o6Lz52PrOzs4n0SbRiAnqHFBJSQlmzZoFRVGgKApUVcXJkye18a6uLthsNkyePBkxMTHIycmB0+kMeI7m5mZkZ2cjOjoaiYmJ2LJlC3p6ekZmb8a4Ww9n4ZcHc3Hfl4T/fTAPvz2cIbslomEJKoCSk5Oxa9cu1NXV4fLly1i0aBFef/11NDY2AgA2b96M48ePo6ysDDU1Nbhz5w5WrFihPb63txfZ2dno7u7G+fPncfjwYRw6dAjbt28f2b0aozp6JqHvRyagQ0dvnNR+iIYrqI9gy5YtC1j/9NNPUVJSggsXLiA5ORlfffUVjhw5gkWLFgEASktLMX36dFy4cAGZmZn44YcfcO3aNfz4448wmUyYM2cOPv74Y7z//vv48MMPodfrR27PxqBnDb8iKqILPmFAVEQXnjX8Q3ZLRMMy5HNAvb29KCsrQ2dnJ1RVRV1dHXw+H6xWq1Yzbdo0WCwW2O12ZGZmwm63Y+bMmTCZTFpNVlYW8vLy0NjYiBdffDGoHm7cuIGYmJih7sKo43A4Hjv+P3+vRnPrbbT5EjEpyoHfnU2PrW9tbcW1a9dGskWiQeno6BhUXdAB1NDQAFVV0dXVhZiYGJSXlyMtLQ319fXQ6/WIi4sLqDeZTNqB5XA4AsKnb7xvbCBerxder1db93g8AAC32z2mzh897hI8APz33/8J4J9BPd/jLtMTPSmDvQASdAC98MILqK+vh9vtxrfffovc3FzU1NQE3WAwiouLsXPnzke2Z2RkQFGUJ/raoXTlypURfb7nnnsOCxYsGNHnJBqMvjcJfyXob0Lr9Xo8//zzSE9PR3FxMWbPno3PP/8cZrO53y/GOZ1OmM1mAIDZbH7kqljfel9Nf4qKiuB2u7WlpaUl2LaJaBQa9q0Yfr8fXq8X6enpiIqKQlVVlTbW1NSE5uZmqKoKAFBVFQ0NDXC5XFpNZWUlFEVBWlragK9hMBi0S/99CxGFv6A+ghUVFWHp0qWwWCxob2/HkSNHcPbsWZw+fRpGoxFr165FYWEh4uPjoSgKNm7cCFVVkZmZCQBYsmQJ0tLSsHr1auzevRsOhwNbt26FzWbjLQNET6GgAsjlcuHtt99Ga2srjEYjZs2ahdOnT+Nvf/sbAGDv3r3Q6XTIycmB1+tFVlYWDhw4oD1+3LhxqKioQF5eHlRVxcSJE5Gbm4uPPhr4doKniV6vH9F3d/xaA412EUIIIbuJYHk8HhiNRrjd7jH1cay9vX1Er1pNmjRpTH1NgcLHYI9R3gs2isTGxiI2NlZ2G0Qhw98HRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKSJlN3AUAghAAAej0dyJ0TUn75js+9YHUhYBtC9e/cAACkpKZI7IaLHaW9vh9FoHHA8LAMoPj4eANDc3PzYnaNAHo8HKSkpaGlpgaIostsJC5yzoRFCoL29HUlJSY+tC8sA0un+derKaDTyP8UQKIrCeQsS5yx4g3lzwJPQRCQNA4iIpAnLADIYDNixYwcMBoPsVsIK5y14nLMnK0L81XUyIqInJCzfARHR2MAAIiJpGEBEJA0DiIikCcsA2r9/P6ZMmYLx48cjIyMDtbW1sluSpri4GPPmzUNsbCwSExOxfPlyNDU1BdR0dXXBZrNh8uTJiImJQU5ODpxOZ0BNc3MzsrOzER0djcTERGzZsgU9PT2h3BVpdu3ahYiICBQUFGjbOGchIsLMsWPHhF6vF19//bVobGwU69atE3FxccLpdMpuTYqsrCxRWloqrl69Kurr68Vrr70mLBaL6Ojo0Go2bNggUlJSRFVVlbh8+bLIzMwUCxYs0MZ7enrEjBkzhNVqFT///LM4ceKESEhIEEVFRTJ2KaRqa2vFlClTxKxZs8SmTZu07Zyz0Ai7AJo/f76w2Wzaem9vr0hKShLFxcUSuxo9XC6XACBqamqEEEK0tbWJqKgoUVZWptVcv35dABB2u10IIcSJEyeETqcTDodDqykpKRGKogiv1xvaHQih9vZ2MXXqVFFZWSlefvllLYA4Z6ETVh/Buru7UVdXB6vVqm3T6XSwWq2w2+0SOxs93G43gP+/Ybeurg4+ny9gzqZNmwaLxaLNmd1ux8yZM2EymbSarKwseDweNDY2hrD70LLZbMjOzg6YG4BzFkphdTPq3bt30dvbG/BDBwCTyYQbN25I6mr08Pv9KCgowMKFCzFjxgwAgMPhgF6vR1xcXECtyWSCw+HQavqb076xsejYsWO4cuUKLl269MgY5yx0wiqA6PFsNhuuXr2Kn376SXYro1pLSws2bdqEyspKjB8/XnY7T7Ww+giWkJCAcePGPXI1wul0wmw2S+pqdMjPz0dFRQXOnDmD5ORkbbvZbEZ3dzfa2toC6v84Z2azud857Rsba+rq6uByufDSSy8hMjISkZGRqKmpwb59+xAZGQmTycQ5C5GwCiC9Xo/09HRUVVVp2/x+P6qqqqCqqsTO5BFCID8/H+Xl5aiurkZqamrAeHp6OqKiogLmrKmpCc3NzdqcqaqKhoYGuFwuraayshKKoiAtLS00OxJCixcvRkNDA+rr67Vl7ty5WLVqlfZvzlmIyD4LHqxjx44Jg8EgDh06JK5duybWr18v4uLiAq5GPE3y8vKE0WgUZ8+eFa2trdry4MEDrWbDhg3CYrGI6upqcfnyZaGqqlBVVRvvu6S8ZMkSUV9fL06dOiWeeeaZp+qS8h+vggnBOQuVsAsgIYT44osvhMViEXq9XsyfP19cuHBBdkvSAOh3KS0t1WoePnwo3n33XTFp0iQRHR0t3njjDdHa2hrwPL/99ptYunSpmDBhgkhISBDvvfee8Pl8Id4bef4cQJyz0OCv4yAiacLqHBARjS0MICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISJr/A/koK/x9eVAXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = torch.load('save/4.DQN_CartPole')\n",
    "\n",
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
